{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/peremartra/Large-Language-Model-Notebooks-Course/blob/main/4-Evaluating%20LLMs/4_3_evaluating_rag_giskard.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div>\n",
        "    <h1>Large Language Models Projects</a></h1>\n",
        "    <h3>Apply and Implement Strategies for Large Language Models</h3>\n",
        "    <h2>4.3-Evaluating LLMs with other LLMs.\n",
        "  </h2>\n",
        "    <h3>Evaluating a RAG solution with Giskard</h3>\n",
        "</div>\n",
        "\n",
        "by [Pere Martra](https://www.linkedin.com/in/pere-martra)\n",
        "\n",
        "____________\n",
        "Models: OpenAI.\n",
        "\n",
        "Colab Environment: CPU-High RAM\n",
        "\n",
        "Keys:\n",
        "* LangChain.\n",
        "* RAG.\n",
        "* Giskard.\n",
        "___________\n",
        "\n",
        "This is the unofficial repository for the book:\n",
        "        <a href=\"https://amzn.to/4eanT1g\"> <b>Large Language Models:</b> Apply and Implement Strategies for Large Language Models</a> (<a href=\"https://link.springer.com/book/10.1007/979-8-8688-0515-8\">Apress</a>).\n",
        "        The book is based on the content of this repository, but the notebooks are being updated, and I am incorporating new examples and chapters.\n",
        "        If you are looking for the official repository for the book, with the original notebooks, you should visit the\n",
        "        <a href=\"https://github.com/Apress/Large-Language-Models-Projects\">Apress repository</a>, where you can find all the notebooks in their original format as they appear in the book."
      ],
      "metadata": {
        "id": "f3cvnSVemTSU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In the final part of this notebook, you will find the necessary code to evaluate the suitability of the responses provided by the Agent using Giskard."
      ],
      "metadata": {
        "id": "DUORXVgTmNYj"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Installing libraries & Loading Dataset"
      ],
      "metadata": {
        "id": "KNW-LniD2I2o"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pva9ehKXUpU2",
        "outputId": "699cf75c-7e27-4676-8305-d0ee21621709"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m400.9/400.9 kB\u001b[0m \u001b[31m23.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m294.6/294.6 kB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.4/76.4 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.0/78.0 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.5/54.5 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.6/49.6 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m383.5/383.5 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m32.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m318.9/318.9 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langchainhub\n",
            "  Downloading langchainhub-0.1.21-py3-none-any.whl.metadata (659 bytes)\n",
            "Requirement already satisfied: packaging<25,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchainhub) (24.1)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchainhub) (2.32.3)\n",
            "Collecting types-requests<3.0.0.0,>=2.31.0.2 (from langchainhub)\n",
            "  Downloading types_requests-2.32.0.20240914-py3-none-any.whl.metadata (1.9 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (2024.8.30)\n",
            "Downloading langchainhub-0.1.21-py3-none-any.whl (5.2 kB)\n",
            "Downloading types_requests-2.32.0.20240914-py3-none-any.whl (15 kB)\n",
            "Installing collected packages: types-requests, langchainhub\n",
            "Successfully installed langchainhub-0.1.21 types-requests-2.32.0.20240914\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m471.6/471.6 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m604.0/604.0 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m56.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m273.8/273.8 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.6/94.6 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m64.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.2/93.2 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.2/13.2 MB\u001b[0m \u001b[31m105.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.5/52.5 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.4/54.4 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.7/63.7 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m93.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m425.7/425.7 kB\u001b[0m \u001b[31m19.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m164.1/164.1 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m26.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.3/49.3 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install -q langchain==0.3.2\n",
        "!pip install -q langchain-openai==0.2.1\n",
        "!pip install -q langchainhub==0.1.21\n",
        "!pip install -q datasets==3.0.1\n",
        "!pip install -q chromadb==0.5.11\n",
        "!pip install -q openai==1.51.0\n",
        "!pip install -q langchain-community==0.3.1"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will download the dataset from the Hugging Face datasets library. It's a dataset with information about diseases."
      ],
      "metadata": {
        "id": "KEwdHiGFkK_S"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "laSDMjqQXuj-",
        "outputId": "368d5afd-3597-4057-a051-4244a8ab1c7a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "4b90b500fe274ce3804d69633e3ce045",
            "f13720f8c0054b37ad63800962c01bd8",
            "a4689ab634be40769ff8c18d3076fbfd",
            "6850ddb41cdb47b3a42117ab825ddf55",
            "def97bfc99c843fcaefeec857804e9a7",
            "2341ca0e2acc46aab7755fab0442d7ec",
            "b6229a3e1b404dc1ae23a13d599f2b1a",
            "ce0ebb7981da496982ecb22864c9a8f8",
            "5174e8de97ad4bf8a6634ff749b61dff",
            "094b862e70a64a8bbe80625e9aacdd77",
            "834b7b72458341949259d352a681a2d5",
            "c3d158dc007941e0bacf1ada9b4d7959",
            "79b2ac0b24bf4dad8babf7b645f8bb84",
            "c933fc46c07440e288dccbd1c349f0a3",
            "1db2aec4c4fe469387537c1ad9eb4137",
            "9a7c82cff6374dadb1185d8a5653fe22",
            "63fac17cc9f444278a3ed1891fb5367f",
            "fc51b487afad4be48fe37f9d5cdf8e09",
            "44ccb08744c54d0d8b1a369b00110241",
            "8aa404c0ddda4b47955ca5b198d63604",
            "0a668f16bf584d65a36a1a1f8b32a045",
            "81591a2f532547e5ac1db6e900e86c3e",
            "6455d0b54a6d4c37b8a4c7e35ad170df",
            "a4fb5033245f46d4b9376e38956bd635",
            "4bb13b7ac5c64ae2a12f03e569320542",
            "be209ee684de4813a63e15c5a562c0f1",
            "9af4090ffa2f4e178f7abda763e091e0",
            "ab796022b81b4ee3b4f105f88f722453",
            "8051d6bec5864a3f82a3949576d51669",
            "16e3ee920a5d4342b93e4e4b8868c700",
            "58bc373738074a51bbe73777881d3179",
            "0b13b00d71ff438da830af5d1b9bc7a0",
            "4fc9af1de5904111a55f93fe0493d003"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "README.md:   0%|          | 0.00/233 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4b90b500fe274ce3804d69633e3ce045"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "medDataset_processed.csv:   0%|          | 0.00/22.5M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c3d158dc007941e0bacf1ada9b4d7959"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/16407 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6455d0b54a6d4c37b8a4c7e35ad170df"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "data = load_dataset(\"keivalya/MedQuad-MedicalQnADataset\", split='train')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "JnWZTcJiXzor",
        "outputId": "3517ef47-d652-4c82-9a3a-715708657f7c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "             qtype                                           Question  \\\n",
              "0   susceptibility  Who is at risk for Lymphocytic Choriomeningiti...   \n",
              "1         symptoms  What are the symptoms of Lymphocytic Choriomen...   \n",
              "2   susceptibility  Who is at risk for Lymphocytic Choriomeningiti...   \n",
              "3  exams and tests  How to diagnose Lymphocytic Choriomeningitis (...   \n",
              "4        treatment  What are the treatments for Lymphocytic Chorio...   \n",
              "5       prevention  How to prevent Lymphocytic Choriomeningitis (L...   \n",
              "6      information          What is (are) Parasites - Cysticercosis ?   \n",
              "7   susceptibility    Who is at risk for Parasites - Cysticercosis? ?   \n",
              "8  exams and tests        How to diagnose Parasites - Cysticercosis ?   \n",
              "9        treatment  What are the treatments for Parasites - Cystic...   \n",
              "\n",
              "                                              Answer  \n",
              "0  LCMV infections can occur after exposure to fr...  \n",
              "1  LCMV is most commonly recognized as causing ne...  \n",
              "2  Individuals of all ages who come into contact ...  \n",
              "3  During the first phase of the disease, the mos...  \n",
              "4  Aseptic meningitis, encephalitis, or meningoen...  \n",
              "5  LCMV infection can be prevented by avoiding co...  \n",
              "6  Cysticercosis is an infection caused by the la...  \n",
              "7  Cysticercosis is an infection caused by the la...  \n",
              "8  If you think that you may have cysticercosis, ...  \n",
              "9  Some people with cysticercosis do not need to ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0e2041a5-09d1-4e51-8fcc-557a485cd19e\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qtype</th>\n",
              "      <th>Question</th>\n",
              "      <th>Answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>susceptibility</td>\n",
              "      <td>Who is at risk for Lymphocytic Choriomeningiti...</td>\n",
              "      <td>LCMV infections can occur after exposure to fr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>symptoms</td>\n",
              "      <td>What are the symptoms of Lymphocytic Choriomen...</td>\n",
              "      <td>LCMV is most commonly recognized as causing ne...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>susceptibility</td>\n",
              "      <td>Who is at risk for Lymphocytic Choriomeningiti...</td>\n",
              "      <td>Individuals of all ages who come into contact ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>exams and tests</td>\n",
              "      <td>How to diagnose Lymphocytic Choriomeningitis (...</td>\n",
              "      <td>During the first phase of the disease, the mos...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>treatment</td>\n",
              "      <td>What are the treatments for Lymphocytic Chorio...</td>\n",
              "      <td>Aseptic meningitis, encephalitis, or meningoen...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>prevention</td>\n",
              "      <td>How to prevent Lymphocytic Choriomeningitis (L...</td>\n",
              "      <td>LCMV infection can be prevented by avoiding co...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>information</td>\n",
              "      <td>What is (are) Parasites - Cysticercosis ?</td>\n",
              "      <td>Cysticercosis is an infection caused by the la...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>susceptibility</td>\n",
              "      <td>Who is at risk for Parasites - Cysticercosis? ?</td>\n",
              "      <td>Cysticercosis is an infection caused by the la...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>exams and tests</td>\n",
              "      <td>How to diagnose Parasites - Cysticercosis ?</td>\n",
              "      <td>If you think that you may have cysticercosis, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>treatment</td>\n",
              "      <td>What are the treatments for Parasites - Cystic...</td>\n",
              "      <td>Some people with cysticercosis do not need to ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0e2041a5-09d1-4e51-8fcc-557a485cd19e')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-0e2041a5-09d1-4e51-8fcc-557a485cd19e button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-0e2041a5-09d1-4e51-8fcc-557a485cd19e');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2654ef24-9599-4c36-9546-f45e93be3af7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2654ef24-9599-4c36-9546-f45e93be3af7')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2654ef24-9599-4c36-9546-f45e93be3af7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "data",
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 16407,\n  \"fields\": [\n    {\n      \"column\": \"qtype\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 16,\n        \"samples\": [\n          \"susceptibility\",\n          \"symptoms\",\n          \"information\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Question\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 14979,\n        \"samples\": [\n          \"What are the symptoms of Danon disease ?\",\n          \"What is (are) Dowling-Degos disease ?\",\n          \"What are the genetic changes related to Pearson marrow-pancreas syndrome ?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Answer\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 15817,\n        \"samples\": [\n          \"These resources address the diagnosis or management of glycogen storage disease type III:  - Gene Review: Gene Review: Glycogen Storage Disease Type III  - Genetic Testing Registry: Glycogen storage disease type III   These resources from MedlinePlus offer information about the diagnosis and management of various health conditions:  - Diagnostic Tests  - Drug Therapy  - Surgery and Rehabilitation  - Genetic Counseling   - Palliative Care\",\n          \"Diagnostic Challenges\\n  \\nFor doctors, diagnosing chronic fatigue syndrome (CFS) can be complicated by a number of factors:\\n  \\n   - There's no lab test or biomarker for CFS.\\n   - Fatigue and other symptoms of CFS are common to many illnesses.\\n   - For some CFS patients, it may not be obvious to doctors that they are ill.\\n   - The illness has a pattern of remission and relapse.\\n   - Symptoms vary from person to person in type, number, and severity.\\n  \\n  \\nThese factors have contributed to a low diagnosis rate. Of the one to four million Americans who have CFS, less than 20% have been diagnosed.\\n  Exams and Screening Tests for CFS\\n  \\nBecause there is no blood test, brain scan, or other lab test to diagnose CFS, the doctor should first rule out other possible causes.\\n  \\nIf a patient has had 6 or more consecutive months of severe fatigue that is reported to be unrelieved by sufficient bed rest and that is accompanied by nonspecific symptoms, including flu-like symptoms, generalized pain, and memory problems, the doctor should consider the possibility that the patient may have CFS. Further exams and tests are needed before a diagnosis can be made:\\n  \\n   - A detailed medical history will be needed and should include a review of medications that could be causing the fatigue and symptoms\\n   - A thorough physical and mental status examination will also be needed\\n   - A battery of laboratory screening tests will be needed to help identify or rule out other possible causes of the symptoms that could be treated\\n   - The doctor may also order additional tests to follow up on results of the initial screening tests\\n  \\n  \\nA CFS diagnosis requires that the patient has been fatigued for 6 months or more and has 4 of the 8 symptoms for CFS for 6 months or more. If, however, the patient has been fatigued for 6 months or more but does not have four of the eight symptoms, the diagnosis may be idiopathic fatigue.\\n  \\nThe complete process for diagnosing CFS can be found here.\\n  \\nAdditional information for healthcare professionals on use of tests can be found here.\",\n          \"Eating, diet, and nutrition have not been shown to play a role in causing or preventing simple kidney cysts.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "data = data.to_pandas()\n",
        "data.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data = data[0:100]"
      ],
      "metadata": {
        "id": "hf7RQa6B5xpx"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "As you can see, the medical information in the dataset is well-organized, and to someone like me, who is not an expert in the field, it appears to be quite valuable. This information could be a useful addition to any general medicine book to support primary care doctors."
      ],
      "metadata": {
        "id": "4FhUslovtiqn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load the langchain libraries to load the document."
      ],
      "metadata": {
        "id": "2sLkrHF6lWhM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import DataFrameLoader\n",
        "from langchain.vectorstores import Chroma"
      ],
      "metadata": {
        "id": "cCBAlIb596wZ"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The Document is in the Answer column, and the others columns are Metadata."
      ],
      "metadata": {
        "id": "nGCkVX6xldOR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_loader = DataFrameLoader(data, page_content_column=\"Answer\")\n"
      ],
      "metadata": {
        "id": "JZX8SaTe99Uf"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_document = df_loader.load()\n",
        "display(df_document[:2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 481
        },
        "id": "HDWnTqRY-IDr",
        "outputId": "65a76c4b-e424-4406-d721-5444802f0e02"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "[Document(metadata={'qtype': 'susceptibility', 'Question': 'Who is at risk for Lymphocytic Choriomeningitis (LCM)? ?'}, page_content='LCMV infections can occur after exposure to fresh urine, droppings, saliva, or nesting materials from infected rodents.  Transmission may also occur when these materials are directly introduced into broken skin, the nose, the eyes, or the mouth, or presumably, via the bite of an infected rodent. Person-to-person transmission has not been reported, with the exception of vertical transmission from infected mother to fetus, and rarely, through organ transplantation.'),\n",
              " Document(metadata={'qtype': 'symptoms', 'Question': 'What are the symptoms of Lymphocytic Choriomeningitis (LCM) ?'}, page_content='LCMV is most commonly recognized as causing neurological disease, as its name implies, though infection without symptoms or mild febrile illnesses are more common clinical manifestations. \\n                \\nFor infected persons who do become ill, onset of symptoms usually occurs 8-13 days after exposure to the virus as part of a biphasic febrile illness. This initial phase, which may last as long as a week, typically begins with any or all of the following symptoms: fever, malaise, lack of appetite, muscle aches, headache, nausea, and vomiting. Other symptoms appearing less frequently include sore throat, cough, joint pain, chest pain, testicular pain, and parotid (salivary gland) pain. \\n                \\nFollowing a few days of recovery, a second phase of illness may occur. Symptoms may consist of meningitis (fever, headache, stiff neck, etc.), encephalitis (drowsiness, confusion, sensory disturbances, and/or motor abnormalities, such as paralysis), or meningoencephalitis (inflammation of both the brain and meninges). LCMV has also been known to cause acute hydrocephalus (increased fluid on the brain), which often requires surgical shunting to relieve increased intracranial pressure. In rare instances, infection results in myelitis (inflammation of the spinal cord) and presents with symptoms such as muscle weakness, paralysis, or changes in body sensation. An association between LCMV infection and myocarditis (inflammation of the heart muscles) has been suggested. \\n                \\nPrevious observations show that most patients who develop aseptic meningitis or encephalitis due to LCMV survive. No chronic infection has been described in humans, and after the acute phase of illness, the virus is cleared from the body. However, as in all infections of the central nervous system, particularly encephalitis, temporary or permanent neurological damage is possible. Nerve deafness and arthritis have been reported. \\n                \\nWomen who become infected with LCMV during pregnancy may pass the infection on to the fetus. Infections occurring during the first trimester may result in fetal death and pregnancy termination, while in the second and third trimesters, birth defects can develop. Infants infected In utero can have many serious and permanent birth defects, including vision problems, mental retardation, and hydrocephaly (water on the brain). Pregnant women may recall a flu-like illness during pregnancy, or may not recall any illness. \\n                \\nLCM is usually not fatal. In general, mortality is less than 1%.')]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can chunk the documents. The size to which we want to split the document is a design decision. The larger it is, the larger the prompt will be, and the slower the Model's response process.\n",
        "\n",
        "We also need to consider the maximum prompt size and ensure that the document does not exceed it."
      ],
      "metadata": {
        "id": "zPuTbSsXl3uF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.text_splitter import CharacterTextSplitter"
      ],
      "metadata": {
        "id": "8wogWV1_-lxh"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_splitter = CharacterTextSplitter(chunk_size=1250, chunk_overlap=100)\n",
        "texts = text_splitter.split_documents(df_document)\n"
      ],
      "metadata": {
        "id": "kg7SRkqO-f9x"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "These warnings we see are because it can't perform the partition of the required size. This is because it waits for a page break to divide the text and does so when possible."
      ],
      "metadata": {
        "id": "y0dn9EfrKoeW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "first_doc = texts[1]\n",
        "print(first_doc.page_content)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XaLWV_TzASSB",
        "outputId": "8d41dee1-012f-4e8c-dc1f-40193f08fc77"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LCMV is most commonly recognized as causing neurological disease, as its name implies, though infection without symptoms or mild febrile illnesses are more common clinical manifestations. \n",
            "                \n",
            "For infected persons who do become ill, onset of symptoms usually occurs 8-13 days after exposure to the virus as part of a biphasic febrile illness. This initial phase, which may last as long as a week, typically begins with any or all of the following symptoms: fever, malaise, lack of appetite, muscle aches, headache, nausea, and vomiting. Other symptoms appearing less frequently include sore throat, cough, joint pain, chest pain, testicular pain, and parotid (salivary gland) pain. \n",
            "                \n",
            "Following a few days of recovery, a second phase of illness may occur. Symptoms may consist of meningitis (fever, headache, stiff neck, etc.), encephalitis (drowsiness, confusion, sensory disturbances, and/or motor abnormalities, such as paralysis), or meningoencephalitis (inflammation of both the brain and meninges). LCMV has also been known to cause acute hydrocephalus (increased fluid on the brain), which often requires surgical shunting to relieve increased intracranial pressure. In rare instances, infection results in myelitis (inflammation of the spinal cord) and presents with symptoms such as muscle weakness, paralysis, or changes in body sensation. An association between LCMV infection and myocarditis (inflammation of the heart muscles) has been suggested. \n",
            "                \n",
            "Previous observations show that most patients who develop aseptic meningitis or encephalitis due to LCMV survive. No chronic infection has been described in humans, and after the acute phase of illness, the virus is cleared from the body. However, as in all infections of the central nervous system, particularly encephalitis, temporary or permanent neurological damage is possible. Nerve deafness and arthritis have been reported. \n",
            "                \n",
            "Women who become infected with LCMV during pregnancy may pass the infection on to the fetus. Infections occurring during the first trimester may result in fetal death and pregnancy termination, while in the second and third trimesters, birth defects can develop. Infants infected In utero can have many serious and permanent birth defects, including vision problems, mental retardation, and hydrocephaly (water on the brain). Pregnant women may recall a flu-like illness during pregnancy, or may not recall any illness. \n",
            "                \n",
            "LCM is usually not fatal. In general, mortality is less than 1%.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B2_Pt7N6Zg2X"
      },
      "source": [
        "### Initialize the Embedding Model and Vector DB"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We load the text-embedding-ada-002 model from OpenAI."
      ],
      "metadata": {
        "id": "n8ROb8oMnRLD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from getpass import getpass\n",
        "import os\n",
        "if not 'OPENAI_API_KEY' in os.environ:\n",
        "  os.environ[\"OPENAI_API_KEY\"] = getpass(\"OpenAI API Key: \")"
      ],
      "metadata": {
        "id": "F_Dn06xGwjKP",
        "outputId": "0464a93d-63ad-48eb-8d46-32850791bb61",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 13,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "OpenAI API Key: ··········\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "U57x2_87YSpb"
      },
      "outputs": [],
      "source": [
        "from langchain_openai import OpenAIEmbeddings\n",
        "\n",
        "model_name = 'text-embedding-ada-002'\n",
        "#model_name = 'text-embedding-3-small'\n",
        "\n",
        "embed = OpenAIEmbeddings(\n",
        "    model=model_name,\n",
        "    #openai_api_key=OPENAI_API_KEY\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The execution of this cell may take 3 to 5 minutes. If you want it to be faster, you can reduce the number of records in the dataset."
      ],
      "metadata": {
        "id": "cgTCwF7UMyNW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "directory_cdb = '/content/drive/MyDrive/chromadb'\n",
        "chroma_db = Chroma.from_documents(\n",
        "    df_document, embed, persist_directory=directory_cdb\n",
        ")"
      ],
      "metadata": {
        "id": "SEhQMQ8eCMj8"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We are going to create three objects.\n",
        "\n",
        "* The language model, which can be any of those from OpenAI, the most common being gpt-3.5.\n",
        "* The memory, responsible for keeping the prompt with all the necessary history.\n",
        "* The retrieval, used to obtain information stored in ChromaDB."
      ],
      "metadata": {
        "id": "KhjV-T8GoarF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "zMRs9Klic5-Y",
        "outputId": "b3988759-8c38-4458-b3ee-bbad9593d871",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-16-b1e61a5f64ec>:9: LangChainDeprecationWarning: Please see the migration guide at: https://python.langchain.com/docs/versions/migrating_memory/\n",
            "  conversational_memory = ConversationBufferWindowMemory(\n"
          ]
        }
      ],
      "source": [
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain_openai import OpenAI\n",
        "from langchain.chains.conversation.memory import ConversationBufferWindowMemory\n",
        "from langchain.chains import RetrievalQA\n",
        "\n",
        "llm=OpenAI(#openai_api_key=OPENAI_API_KEY,\n",
        "           temperature=0.0)\n",
        "\n",
        "conversational_memory = ConversationBufferWindowMemory(\n",
        "    memory_key='chat_history',\n",
        "    k=4, #Number of messages stored in memory\n",
        "    return_messages=True #Must return the messages in the response.\n",
        ")\n",
        "\n",
        "qa = RetrievalQA.from_chain_type(\n",
        "    llm=llm,\n",
        "    chain_type=\"stuff\",\n",
        "    retriever=chroma_db.as_retriever()\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can try the isolated Retrieval to see if the information it returns is relevant.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "ergrieE4o8qu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LaYSq0V-dxHw",
        "outputId": "99e89396-1ba4-4c46-bfdc-c9e851d4aebd"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'query': 'What is the main symptom of LCM?',\n",
              " 'result': ' The main symptom of LCM is a biphasic febrile illness, which includes symptoms such as fever, malaise, lack of appetite, muscle aches, headache, nausea, and vomiting.'}"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ],
      "source": [
        "qa.invoke(\"What is the main symptom of LCM?\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perfect! The information returned is exactly what we desired.\n",
        "\n",
        "## Creating the Agent."
      ],
      "metadata": {
        "id": "pf9MXPeipEBO"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "FwCYrS4duqBW"
      },
      "outputs": [],
      "source": [
        "from langchain.agents import Tool, AgentExecutor\n",
        "DESCRIPTION = 'use this tool when answering medical knowledge queries to get more information about the topic'\n",
        "#DESCRIPTION = 'Use this tool if any part of the QUERY or CHAT HISTORY history discusses concepts related to medicine or diseases to get more information about the topic'\n",
        "\n",
        "#Defining the list of tool objects to be used by LangChain.\n",
        "tools = [\n",
        "    Tool(\n",
        "        name='Medical KB',\n",
        "        func=qa.run,\n",
        "        description=(\n",
        "            DESCRIPTION\n",
        "        )\n",
        "    )\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "JaKTzPUEvOoy",
        "outputId": "787a391e-8aa6-49a4-c59c-a2e0ca341790",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/langsmith/client.py:354: LangSmithMissingAPIKeyWarning: API key must be provided when using hosted LangSmith API\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from langchain.agents import initialize_agent, create_react_agent\n",
        "from langchain import hub\n",
        "\n",
        "prompt = hub.pull(\"hwchase17/react-chat\")\n",
        "agent = create_react_agent(\n",
        "    #agent='chat-conversational-react-description',\n",
        "    tools=tools,\n",
        "    llm=llm,\n",
        "    prompt=prompt,\n",
        "    #verbose=True,\n",
        "    #max_iterations=3,\n",
        "    #early_stopping_method='generate'\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create an agent executor by passing in the agent and tools\n",
        "agent_executor = AgentExecutor(agent=agent,\n",
        "                               tools=tools,\n",
        "                               verbose=True,\n",
        "                               memory=conversational_memory,\n",
        "                               max_iterations=30,\n",
        "                               max_execution_time=600,\n",
        "                               #early_stopping_method='generate',\n",
        "                               handle_parsing_errors=True\n",
        "                               )"
      ],
      "metadata": {
        "id": "5sjTL1EU1vNW"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IlxUBWKcvzeP"
      },
      "source": [
        "### Using the Conversational Agent"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZZapCP4Pv2kz"
      },
      "source": [
        "To make queries we simply call the `agent` directly.\n",
        "\n",
        "First i will try a order not related to the Medical field."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "85vipqC02deV",
        "outputId": "c02aba7a-6fca-41df-826b-d2a55a6a6ebf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Area of square\u001b[0m\u001b[36;1m\u001b[1;3m I don't know.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: The area of a square with sides of 2 units is 4 square units.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input': 'Give me the area of square of 2x2',\n",
              " 'chat_history': [],\n",
              " 'output': 'The area of a square with sides of 2 units is 4 square units.'}"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ],
      "source": [
        "agent_executor.invoke({\"input\": \"Give me the area of square of 2x2\"})"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perfect, the model has responded without accessing the configured knowledge database.\n",
        "\n",
        "Now I will try with a question that is also not related to health."
      ],
      "metadata": {
        "id": "8suStMR7G11e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "agent_executor.invoke({\"input\": \"Do you know who is Clark Kent?\"})"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9YEsxEuCVgMv",
        "outputId": "95ab9e3f-fc2d-48e2-a7c4-b92aaebcb31e"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Clark Kent\u001b[0m\u001b[36;1m\u001b[1;3m I don't know.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: Clark Kent is the secret identity of Superman, a fictional superhero appearing in American comic books published by DC Comics. The character was created by writer Jerry Siegel and artist Joe Shuster, and first appeared in the comic book Action Comics #1 (cover-dated June 1938 and published April 18, 1938).\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input': 'Do you know who is Clark Kent?',\n",
              " 'chat_history': [HumanMessage(content='Give me the area of square of 2x2', additional_kwargs={}, response_metadata={}),\n",
              "  AIMessage(content='The area of a square with sides of 2 units is 4 square units.', additional_kwargs={}, response_metadata={})],\n",
              " 'output': 'Clark Kent is the secret identity of Superman, a fictional superhero appearing in American comic books published by DC Comics. The character was created by writer Jerry Siegel and artist Joe Shuster, and first appeared in the comic book Action Comics #1 (cover-dated June 1938 and published April 18, 1938).'}"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "It has not accessed either, as the model has been able to identify that it is not a question related to the database that LangChain provides.\n",
        "\n",
        "Now it's time to try with a question related to Medicine. Let's see if the model can understand that it should first look for information in the vector database at its disposal."
      ],
      "metadata": {
        "id": "lyd0XJ3CHjVb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        " agent_executor.memory.clear()"
      ],
      "metadata": {
        "id": "Wtwgfuy158LV"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RJoAhy76vzAB",
        "outputId": "d4148000-d8da-47ff-d13d-e3f142b5c2f3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Botulism\u001b[0m\u001b[36;1m\u001b[1;3m Botulism is a rare but serious paralytic illness caused by a nerve toxin produced by certain bacteria. It can be contracted through contaminated food, wounds, or ingestion of bacterial spores. Symptoms include muscle paralysis, difficulty swallowing, and respiratory failure. Treatment includes antitoxin, supportive care, and removal of the source of the toxin.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: To confirm a diagnosis of botulism, a doctor may perform a physical exam, review symptoms and medical history, and order laboratory tests such as a stool or blood test. They may also perform a nerve conduction study or electromyography to assess muscle function. It is important to seek medical attention immediately if botulism is suspected.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input': 'I have a patient that can have Botulism,\\nhow can I confirm the diagnose?',\n",
              " 'chat_history': [],\n",
              " 'output': 'To confirm a diagnosis of botulism, a doctor may perform a physical exam, review symptoms and medical history, and order laboratory tests such as a stool or blood test. They may also perform a nerve conduction study or electromyography to assess muscle function. It is important to seek medical attention immediately if botulism is suspected.'}"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ],
      "source": [
        "agent_executor.invoke({\"input\": \"\"\"I have a patient that can have Botulism,\n",
        "how can I confirm the diagnose?\"\"\"})"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perfect, the most important thing for us is that it has been able to identify that it should go to the medical database to search for information about the symptoms."
      ],
      "metadata": {
        "id": "8lsaoF8nJNnR"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mQeicHTj2pmY",
        "outputId": "5b5de623-2e41-4c29-dc11-a676f2573489"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? No\n",
            "Final Answer: Yes, botulism is a serious illness that requires immediate medical attention. It is caused by a toxin produced by the bacteria Clostridium botulinum and can lead to paralysis and even death if left untreated. It is important to seek medical attention immediately if botulism is suspected.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input': 'Is this an important illness?',\n",
              " 'chat_history': [HumanMessage(content='I have a patient that can have Botulism,\\nhow can I confirm the diagnose?', additional_kwargs={}, response_metadata={}),\n",
              "  AIMessage(content='To confirm a diagnosis of botulism, a doctor may perform a physical exam, review symptoms and medical history, and order laboratory tests such as a stool or blood test. They may also perform a nerve conduction study or electromyography to assess muscle function. It is important to seek medical attention immediately if botulism is suspected.', additional_kwargs={}, response_metadata={})],\n",
              " 'output': 'Yes, botulism is a serious illness that requires immediate medical attention. It is caused by a toxin produced by the bacteria Clostridium botulinum and can lead to paralysis and even death if left untreated. It is important to seek medical attention immediately if botulism is suspected.'}"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ],
      "source": [
        "agent_executor.invoke({\"input\": \"Is this an important illness?\"})"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "And the memory works perfectly. We can maintain a conversation, taking into account that the model knows the previous questions and answers."
      ],
      "metadata": {
        "id": "quf3qyI_aVz3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluating the solution with Giskard\n",
        "\n",
        "Install and Load the libraries."
      ],
      "metadata": {
        "id": "FQwHcAMi3e85"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q giskard[llm]==2.15.1\n",
        "from giskard.rag import KnowledgeBase, generate_testset, evaluate"
      ],
      "metadata": {
        "id": "6Em-qkfK637x",
        "outputId": "2f8ad0d6-2238-4206-a31d-f96019ee5e35",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.4/60.4 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m981.5/981.5 kB\u001b[0m \u001b[31m17.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m544.9/544.9 kB\u001b[0m \u001b[31m24.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.1/61.1 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.0/84.0 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.0/27.0 MB\u001b[0m \u001b[31m88.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.8/140.8 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m113.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.3/143.3 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m36.4/36.4 MB\u001b[0m \u001b[31m54.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.7/85.7 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m106.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m563.0/563.0 kB\u001b[0m \u001b[31m30.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.9/56.9 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for docopt (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Is necesary to create a Dataframe with just the column containing the information used to create the RAG system."
      ],
      "metadata": {
        "id": "qXN4hGqDoMg1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "df_giskard = pd.DataFrame([d.page_content for d in df_document], columns=[\"text\"])\n",
        "df_giskard.head()"
      ],
      "metadata": {
        "id": "yeIMrHlhMJDF",
        "outputId": "a78935d6-2971-411e-d6e3-b1843edc0c6c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text\n",
              "0  LCMV infections can occur after exposure to fr...\n",
              "1  LCMV is most commonly recognized as causing ne...\n",
              "2  Individuals of all ages who come into contact ...\n",
              "3  During the first phase of the disease, the mos...\n",
              "4  Aseptic meningitis, encephalitis, or meningoen..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-76cea786-ebb9-40ba-b394-b076f402ba7a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>LCMV infections can occur after exposure to fr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>LCMV is most commonly recognized as causing ne...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Individuals of all ages who come into contact ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>During the first phase of the disease, the mos...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Aseptic meningitis, encephalitis, or meningoen...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-76cea786-ebb9-40ba-b394-b076f402ba7a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-76cea786-ebb9-40ba-b394-b076f402ba7a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-76cea786-ebb9-40ba-b394-b076f402ba7a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-edd98e4e-0800-4af9-a227-9a00449c19a6\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-edd98e4e-0800-4af9-a227-9a00449c19a6')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-edd98e4e-0800-4af9-a227-9a00449c19a6 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_giskard",
              "summary": "{\n  \"name\": \"df_giskard\",\n  \"rows\": 100,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 100,\n        \"samples\": [\n          \"The respiratory failure and paralysis that occur with severe botulism may require a patient to be on a breathing machine (ventilator) for weeks or months, plus intensive medical and nursing care. The paralysis slowly improves. Botulism can be treated with an antitoxin which blocks the action of toxin circulating in the blood. Antitoxin for infants is available from the California Department of Public Health, and antitoxin for older children and adults is available through CDC.If given before paralysis is complete, antitoxin can prevent worsening and shorten recovery time. Physicians may try to remove contaminated food still in the gut by inducing vomiting or by using enemas. Wounds should be treated, usually surgically, to remove the source of the toxin-producing bacteria followed by administration of appropriate antibiotics. Good supportive care in a hospital is the mainstay of therapy for all forms of botulism.\",\n          \"Suggested General Guidelines \\n  \\n    \\n   \\n  \\nIt is important to remember that the first time a person gets scabies they usually have no symptoms during the first 2 to 6 weeks they are infested; however they can still spread scabies during this time. \\n  \\nTreatment should be given to both the infested person and to household members and sexual contacts, particularly those who have had prolonged direct skin-to-skin contact with the infested person. Both sexual and close personal contacts who have had direct prolonged skin-to-skin contact with an infested person within the preceding month should be examined and treated. All persons should be treated at the same time to prevent reinfestation. Scabies may sometimes be sexually-acquired in adults, but is rarely sexually-acquired in children. \\n  \\nBedding, clothing, and towels used by infested persons or their household, sexual, and close contacts (as defined above) anytime during the three days before treatment should be decontaminated by washing in hot water and drying in a hot dryer, by dry-cleaning, or by sealing in a plastic bag for at least 72 hours. Scabies mites generally do not survive more than 2 to 3 days away from human skin. \\n  \\nUse of insecticide sprays and fumigants is not recommended. \\n    \\n  Medications Used to Treat Scabies \\n  \\n    \\n   \\n  \\nProducts used to treat scabies are called scabicides because they kill scabies mites; some also kill mite eggs. Scabicides used to treat human scabies are available only with a doctor\\u2019s prescription. No \\u201cover-the-counter\\u201d (non-prescription) products have been tested and approved to treat scabies. \\n  \\nScabicide should be applied to all areas of the body from the neck down to the feet and toes. In addition, when treating infants and young children, scabicide also should be applied to their entire head and neck because scabies can affect their face, scalp, and neck, as well as the rest of their body. The scabicide should be applied to a clean body and left on for the recommended time before washing it off. Clean clothing should be worn after treatment. \\n  \\nThe instructions contained in the box or printed on the label always should be followed carefully. Always contact a doctor or pharmacist if unsure how to use a particular medicine. \\n  \\nBecause the symptoms of scabies are due to a hypersensitivity reaction (allergy) to mites and their feces (scybala), itching still may continue for several weeks after treatment even if all the mites and eggs are killed. If itching still is present more than 2 to 4 weeks after treatment or if new burrows or pimple-like rash lesions continue to appear, retreatment may be necessary. \\n  \\nSkin sores that become infected should be treated with an appropriate antibiotic prescribed by a doctor.\",\n          \"Baylisascaris infection can be prevented by avoiding contact with raccoons and their feces. Washing your hands after working or playing outdoors is good practice for preventing a number of diseases.\\n    \\nDo not keep, feed, or adopt wild animals, including raccoons, as pets. Infection rarely causes symptoms in raccoons, so you cannot tell if a raccoon is infected by observing its behavior. Roundworm eggs passed in the feces of infected raccoons are not visible to the naked eye. Eggs can only be seen using a microscope.\\n    \\n     \\n    \\n    \\nYou may discourage raccoons from living in and around your home or parks by taking these steps:\\n    \\n     - prevent access to food\\n     - keep trash containers tightly closed\\n     - close off access to attics and basements\\n     - keep sandboxes covered when not in use (raccoons may use sandboxes as a latrine)\\n     - remove fish ponds -- they eat the fish and drink the water\\n     - eliminate water sources\\n     - remove bird feeders\\n     - clear brush so raccoons are not likely to make a den on your property\\n    \\n    \\nStay away from areas and materials that might be contaminated by raccoon feces. Raccoons typically defecate at the base of or in raised forks of trees, or on raised horizontal surfaces such as fallen logs, stumps, or large rocks. Raccoon feces also can be found on woodpiles, decks, rooftops, and in attics, garages, and haylofts. Feces usually are dark and tubular, have a pungent odor (usually worse than dog or cat feces), and often contain undigested seeds or other food items.\\n    \\nIf you have found a raccoon latrine near your home, cleaning the area may prevent possible infection. Newly deposited eggs take at least 2-4 weeks to become infective. Prompt removal and destruction of raccoon feces will reduce risk for exposure and possible infection.\\n    \\nMore on: Raccoon Latrine Clean-up [PDF, 111 KB, 1 page]\\n    \\n     \\n    \\n    \\nIf you choose to clean the site yourself, care should be taken to avoid contaminating hands and clothes.\\n    \\n     - Wear disposable gloves to help prevent cross contamination.\\n     - Wear a N95-rated respirator if working in a confined space to prevent accidental ingestion of eggs or other harmful materials.\\n     - Avoid stirring up dust and debris- you can lightly mist the latrine area with a little water from a spray bottle to reduce the amount of dust.\\n     - Wear rubber boots that can be scrubbed or cover your shoes with disposable booties that can be thrown away, so that you do not bring eggs into your household.\\n     - Feces and material contaminated with raccoon feces should be removed and burned, buried, or sent to a landfill.\\n     - Most chemicals do not kill roundworm eggs; however, heat kills the eggs instantly.\\n     - Treat feces-soiled decks, patios, and other surfaces with boiling water or a propane torch (please contact your local fire department for regulations and safety practices).\\n    \\n    \\nTo help further reduce the risk of possible infection, wash your hands well with soap and warm running water. Clean/launder your clothes thoroughly with hot water and detergent.\\n    \\nMore on: Handwashing\\n    \\nIf you are cleaning an indoor raccoon latrine and are not able to use a propane torch, use a damp (but not wet) sponge to wipe the area with hot soapy water. Rinse your sponge frequently. After you are finished, flush dirty water down the toilet. Place the sponge in a plastic bag and put the plastic bag in the garbage.\\n    \\nContact your local animal control office for additional assistance.\\n    Dogs\\n    \\nDogs may be infected with adult B. procyonis roundworms, but may not show symptoms. Have all pets de-wormed under a veterinarian's supervision and take precautions to avoid contact with their feces.\\n    Exotic pets\\n    \\nRaccoons and dogs are not the only hosts of Baylisascaris. B. procyonis infection has also been documented in kinkajous. Other animals such as coatis may be susceptible. When wild animals are kept as pets, there can be a risk of disease transmission to humans.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Using the information from the dataset, we ask Giskard to create a Knowledge Base, which is nothing more than a set of questions along with their respective answers. Both the questions and answers are generated by OpenAI's most advanced model, which is why it requires our OpenAI key to be provided."
      ],
      "metadata": {
        "id": "VK3hgB-MoWLj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "kb_giskard = KnowledgeBase(df_giskard)"
      ],
      "metadata": {
        "id": "iPaw4w9c_Gbo"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "kb_giskard"
      ],
      "metadata": {
        "id": "xewqzBgzwrOv",
        "outputId": "3ca334e8-fe78-44fb-d10a-89ee1f531f8e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<giskard.rag.knowledge_base.KnowledgeBase at 0x7a9b6e95cc70>"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The more questions you generate, the more you will be charged.\n",
        "test_questions = generate_testset(\n",
        "    kb_giskard,\n",
        "    num_questions=10,\n",
        "    agent_description=\"Medical assistant for diagnosis and treatment support.\",\n",
        ")"
      ],
      "metadata": {
        "id": "VcSnDxElKQmw",
        "outputId": "8b44734a-0365-4d5b-84b5-cf1daf1afa14",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85,
          "referenced_widgets": [
            "402bc040234745f8ae9a557728eb8354",
            "c1409fb9cd114ea1bcc5dd77ddc504e7",
            "1e3b95759e8a46e693599b340a0835f9",
            "4038dab690dc4e17a0123cd8bc07edcd",
            "04cdfa01853447449e6e58ae355e0687",
            "332819930be54450b2c4c66aa37fb3cd",
            "4e03497cf82e44b7bc1135d73e91d4f0",
            "82b885247c6f4cb2a826b6abc6d5175e",
            "d94c38c92b444e92b6e8a08c4c652e64",
            "ceb438b0126847ff9009604005732812",
            "372c3c4bcbf643ce9141a987125d8206"
          ]
        }
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:giskard.rag:Finding topics in the knowledge base.\n",
            "INFO:giskard.rag:Found 5 topics in the knowledge base.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating questions:   0%|          | 0/10 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "402bc040234745f8ae9a557728eb8354"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_test_questions = test_questions.to_pandas()"
      ],
      "metadata": {
        "id": "rYH3J0MXTJl1"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_test_questions.head()"
      ],
      "metadata": {
        "id": "rvHhO_oDThft",
        "outputId": "b4778f6b-69f8-43ac-e65d-0d60e1ffcf56",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        }
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                               question  \\\n",
              "id                                                                                        \n",
              "04ef0415-5c30-4265-941f-9d65790e9f34  What are some of the ways to prevent whipworm ...   \n",
              "4b435d83-6fa1-4ff9-bca6-9414971db308  What are the possible treatments for cysticerc...   \n",
              "83b3c29a-796d-4440-b38b-02a40662f506  In order to rule out other conditions similar ...   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50  What are some specific actions that can be und...   \n",
              "3ba564fe-936d-4836-bdfb-dbe0744bd2eb  Considering the context of body lice infestati...   \n",
              "\n",
              "                                                                       reference_answer  \\\n",
              "id                                                                                        \n",
              "04ef0415-5c30-4265-941f-9d65790e9f34  To prevent whipworm infection, avoid ingesting...   \n",
              "4b435d83-6fa1-4ff9-bca6-9414971db308  There are medications available to treat cysti...   \n",
              "83b3c29a-796d-4440-b38b-02a40662f506  Tests for botulism may include a brain scan, s...   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50  Preventative measures include avoiding contact...   \n",
              "3ba564fe-936d-4836-bdfb-dbe0744bd2eb  The five main kinds of botulism are foodborne ...   \n",
              "\n",
              "                                                                      reference_context  \\\n",
              "id                                                                                        \n",
              "04ef0415-5c30-4265-941f-9d65790e9f34  Document 15: The best way to prevent whipworm ...   \n",
              "4b435d83-6fa1-4ff9-bca6-9414971db308  Document 9: Some people with cysticercosis do ...   \n",
              "83b3c29a-796d-4440-b38b-02a40662f506  Document 82: Physicians may consider the diagn...   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50  Document 67: Raccoons are the primary, or defi...   \n",
              "3ba564fe-936d-4836-bdfb-dbe0744bd2eb  Document 80: In the United States, an average ...   \n",
              "\n",
              "                                     conversation_history  \\\n",
              "id                                                          \n",
              "04ef0415-5c30-4265-941f-9d65790e9f34                   []   \n",
              "4b435d83-6fa1-4ff9-bca6-9414971db308                   []   \n",
              "83b3c29a-796d-4440-b38b-02a40662f506                   []   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50                   []   \n",
              "3ba564fe-936d-4836-bdfb-dbe0744bd2eb                   []   \n",
              "\n",
              "                                                                               metadata  \n",
              "id                                                                                       \n",
              "04ef0415-5c30-4265-941f-9d65790e9f34  {'question_type': 'simple', 'seed_document_id'...  \n",
              "4b435d83-6fa1-4ff9-bca6-9414971db308  {'question_type': 'simple', 'seed_document_id'...  \n",
              "83b3c29a-796d-4440-b38b-02a40662f506  {'question_type': 'complex', 'seed_document_id...  \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50  {'question_type': 'complex', 'seed_document_id...  \n",
              "3ba564fe-936d-4836-bdfb-dbe0744bd2eb  {'question_type': 'distracting element', 'seed...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ecfb4c9a-6d67-4f3b-90d2-e74a38d0593c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>reference_answer</th>\n",
              "      <th>reference_context</th>\n",
              "      <th>conversation_history</th>\n",
              "      <th>metadata</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>04ef0415-5c30-4265-941f-9d65790e9f34</th>\n",
              "      <td>What are some of the ways to prevent whipworm ...</td>\n",
              "      <td>To prevent whipworm infection, avoid ingesting...</td>\n",
              "      <td>Document 15: The best way to prevent whipworm ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'simple', 'seed_document_id'...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4b435d83-6fa1-4ff9-bca6-9414971db308</th>\n",
              "      <td>What are the possible treatments for cysticerc...</td>\n",
              "      <td>There are medications available to treat cysti...</td>\n",
              "      <td>Document 9: Some people with cysticercosis do ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'simple', 'seed_document_id'...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>83b3c29a-796d-4440-b38b-02a40662f506</th>\n",
              "      <td>In order to rule out other conditions similar ...</td>\n",
              "      <td>Tests for botulism may include a brain scan, s...</td>\n",
              "      <td>Document 82: Physicians may consider the diagn...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'complex', 'seed_document_id...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65c3ad44-0282-4212-b7a6-1d09becfec50</th>\n",
              "      <td>What are some specific actions that can be und...</td>\n",
              "      <td>Preventative measures include avoiding contact...</td>\n",
              "      <td>Document 67: Raccoons are the primary, or defi...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'complex', 'seed_document_id...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3ba564fe-936d-4836-bdfb-dbe0744bd2eb</th>\n",
              "      <td>Considering the context of body lice infestati...</td>\n",
              "      <td>The five main kinds of botulism are foodborne ...</td>\n",
              "      <td>Document 80: In the United States, an average ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'distracting element', 'seed...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ecfb4c9a-6d67-4f3b-90d2-e74a38d0593c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ecfb4c9a-6d67-4f3b-90d2-e74a38d0593c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ecfb4c9a-6d67-4f3b-90d2-e74a38d0593c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c99c969b-d478-4635-98b0-5f0e0115d5f7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c99c969b-d478-4635-98b0-5f0e0115d5f7')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c99c969b-d478-4635-98b0-5f0e0115d5f7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_test_questions",
              "summary": "{\n  \"name\": \"df_test_questions\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"ea3b30cd-3294-42f9-8d2c-d6ed7416ad38\",\n          \"4b435d83-6fa1-4ff9-bca6-9414971db308\",\n          \"22172cef-3ac7-4be4-a48b-01547e278ce5\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"question\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"How is rabies diagnosed in animals and humans and why is rapid and accurate laboratory diagnosis essential?\",\n          \"What are the possible treatments for cysticercosis?\",\n          \"What are the three main ways Toxoplasmosis can be transmitted, considering that botulism, another disease, can result in long-term health complications and even death?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"reference_answer\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 9,\n        \"samples\": [\n          \"Rabies is diagnosed in animals using the direct fluorescent antibody (DFA) test, which looks for the presence of rabies virus antigens in brain tissue, while in humans, several tests are required. Rapid and accurate laboratory diagnosis is essential for timely administration of postexposure prophylaxis, potentially saving a patient from unnecessary physical and psychological trauma, and financial burdens, if the animal is not rabid.\",\n          \"There are medications available to treat cysticercosis, and sometimes surgery may be needed.\",\n          \"The three principal routes of transmission for Toxoplasmosis are foodborne transmission, animal-to-human (zoonotic) transmission, and mother-to-child (congenital) transmission.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"reference_context\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 9,\n        \"samples\": [\n          \"Document 78: In animals, rabies is diagnosed using the direct fluorescent antibody (DFA) test, which looks for the presence of rabies virus antigens in brain tissue. In humans, several tests are required.\\n \\nRapid and accurate laboratory diagnosis of rabies in humans and other animals is essential for timely administration of postexposure prophylaxis. Within a few hours, a diagnostic laboratory can determine whether or not an animal is rabid and inform the responsible medical personnel. The laboratory results may save a patient from unnecessary physical and psychological trauma, and financial burdens, if the animal is not rabid.\\n \\nIn addition, laboratory identification of positive rabies cases may aid in defining current epidemiologic patterns of disease and provide appropriate information for the development of rabies control programs.\\n \\nThe nature of rabies disease dictates that laboratory tests be standardized, rapid, sensitive, specific, economical, and reliable.\",\n          \"Document 9: Some people with cysticercosis do not need to be treated. There are medications available to treat cysticercosis for those who do need treatment. Sometimes surgery may be needed. Your doctor will advise you on which treatment is best for you.\\n    \\nMore on: Resources for Health Professionals: Treatment\\n    \\nMore on: Taeniasis\",\n          \"Document 99: Toxoplasmosis is caused by the protozoan parasite Toxoplasma gondii. In the United States it is estimated that 22.5% of the population 12 years and older have been infected with Toxoplasma. In various places throughout the world, it has been shown that up to 95% of some populations have been infected with Toxoplasma. Infection is often highest in areas of the world that have hot, humid climates and lower altitudes. \\n    \\nToxoplasmosis is not passed from person-to-person, except in instances of mother-to-child (congenital) transmission and blood transfusion or organ transplantation. People typically become infected by three principal routes of transmission. \\n         \\n     \\n     \\n       \\n      \\n     \\n    \\n      \\n    Foodborne transmission \\n    \\nThe tissue form of the parasite (a microscopic cyst consisting of bradyzoites) can be transmitted to humans by food. People become infected by: \\n     \\n     - Eating undercooked, contaminated meat (especially pork, lamb, and venison) \\n     - Accidental ingestion of undercooked, contaminated meat after handling it and not washing hands thoroughly (Toxoplasma cannot be absorbed through intact skin) \\n     - Eating food that was contaminated by knives, utensils, cutting boards, or other foods that had contact with raw, contaminated meat \\n     \\n    \\n      \\n    Animal-to-human (zoonotic) transmission \\n    \\nCats play an important role in the spread of toxoplasmosis. They become infected by eating infected rodents, birds, or other small animals. The parasite is then passed in the cat's feces in an oocyst form, which is microscopic. \\n    \\nKittens and cats can shed millions of oocysts in their feces for as long as 3 weeks after infection. Mature cats are less likely to shed Toxoplasma if they have been previously infected. A Toxoplasma-infected cat that is shedding the parasite in its feces contaminates the litter box. If the cat is allowed outside, it can contaminate the soil or water in the environment as well. \\n     \\n       \\n      \\n     \\n    \\nPeople can accidentally swallow the oocyst form of the parasite. People can be infected by: \\n     \\n     - Accidental ingestion of oocysts after cleaning a cat's litter box when the cat has shed Toxoplasma in its feces \\n     - Accidental ingestion of oocysts after touching or ingesting anything that has come into contact with a cat's feces that contain Toxoplasma \\n     - Accidental ingestion of oocysts in contaminated soil (e.g., not washing hands after gardening or eating unwashed fruits or vegetables from a garden) \\n     - Drinking water contaminated with the Toxoplasma parasite \\n     \\n    \\n      \\n    Mother-to-child (congenital) transmission \\n    \\nA woman who is newly infected with Toxoplasma during pregnancy can pass the infection to her unborn child (congenital infection). The woman may not have symptoms, but there can be severe consequences for the unborn child, such as diseases of the nervous system and eyes. \\n    \\n      \\n    Rare instances of transmission \\n    \\nOrgan transplant recipients can become infected by receiving an organ from a Toxoplasma-positive donor. Rarely, people can also become infected by receiving infected blood via transfusion. Laboratory workers who handle infected blood can also acquire infection through accidental inoculation.\\n\\nDocument 98: A single-celled parasite called Toxoplasma gondii causes a disease known as toxoplasmosis. While the parasite is found throughout the world, more than 60 million people in the United States may be infected with the Toxoplasma parasite. Of those who are infected, very few have symptoms because a healthy person\\u2019s immune system usually keeps the parasite from causing illness. However, pregnant women and individuals who have compromised immune systems should be cautious; for them, a Toxoplasma infection could cause serious health problems.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"conversation_history\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"metadata\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_test_questions['question'].iloc[1]\n"
      ],
      "metadata": {
        "id": "i1nOfJ7pSEKg",
        "outputId": "457b6272-615a-4a3c-d06b-58dc30095485",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'What are the possible treatments for cysticercosis?'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_test_questions['reference_answer'].iloc[1]\n"
      ],
      "metadata": {
        "id": "_nzEw9rNTuW2",
        "outputId": "de0f9700-358f-4aed-f5fd-c2274738907c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'There are medications available to treat cysticercosis, and sometimes surgery may be needed.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_test_questions['reference_context'].iloc[1]"
      ],
      "metadata": {
        "id": "RcT53F6ZTvcs",
        "outputId": "ae54b62e-518d-48cd-a53a-e6475efe354b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Document 9: Some people with cysticercosis do not need to be treated. There are medications available to treat cysticercosis for those who do need treatment. Sometimes surgery may be needed. Your doctor will advise you on which treatment is best for you.\\n    \\nMore on: Resources for Health Professionals: Treatment\\n    \\nMore on: Taeniasis'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "A function is created that will be called from Giskard's evaluate function. This function receives the question and returns the agent's response."
      ],
      "metadata": {
        "id": "21tFRKfXC2wF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def use_agent(question, history=None):\n",
        "  answer=agent_executor.invoke({\"input\": question})\n",
        "  return answer['output']"
      ],
      "metadata": {
        "id": "InRbIK2_V2HU"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "agent_executor.memory.clear()"
      ],
      "metadata": {
        "id": "DRuhL8XMXK-I"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "report = evaluate(use_agent, testset=test_questions, knowledge_base=kb_giskard)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "0f0e0eb678fb4561ae67c8a56d0024f8",
            "f94992c7afee4b1591db5164cf107880",
            "9e00a1effe084ab9806fb7aab4775984",
            "cd7396bfa8df48b2864a109ce7c66a2c",
            "8f5ef7b50a3c43609c7c1a168e6ac9b6",
            "7fce24a65eb74ec8bec35caefedc3107",
            "cad8096da6874fe0898f1345c1e7bbf4",
            "3b083846d432457782171a810b460771",
            "ae34ed2aae744624b048ec526413dbdf",
            "6b8a6330b0b74c649176611711b99231",
            "f3ad037f417c46b5a3e52e86486c379a",
            "ddafa25f88ca47fb81172779c34ca1aa",
            "2885e20894324970a12f52d3cd7f15b1",
            "98ffcb3dc9bf4cf6bdb8a00e57f73a55",
            "6f234f8281c4445792dc4ea0ce7a7b9e",
            "840a177b18254751b07f564176429e36",
            "9806352a846a4fec9c6fecd2aa3fc1ad",
            "c024e5f9f06342169b7f7c4a5327d7ff",
            "a66f8156f0d6406cab082b74b1b74aa2",
            "b023814d0cf04b589926d9f64cd72ad3",
            "2ec7a5d413864daab9f68dbdd30b04b2",
            "33297c2c5e1d4ad0a3fe95d7bff4cb46"
          ]
        },
        "id": "D3fm-_7RU4y9",
        "outputId": "acadcb6c-cd92-4367-d4d5-c0dcd0c5a6ed"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Asking questions to the agent:   0%|          | 0/10 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0f0e0eb678fb4561ae67c8a56d0024f8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Whipworm infection\u001b[0m\u001b[36;1m\u001b[1;3m The best way to prevent whipworm infection is to avoid ingesting soil contaminated with human feces, wash hands before handling food, teach children about handwashing, and wash, peel, or cook all raw vegetables and fruits. Additionally, transmission of infection to others can be prevented by not defecating outdoors and having effective sewage disposal systems.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: Some ways to prevent whipworm infection include avoiding ingesting contaminated soil, washing hands before handling food, and properly washing, peeling, or cooking raw fruits and vegetables. It is also important to not defecate outdoors and have proper sewage disposal systems.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: cysticercosis\u001b[0m\u001b[36;1m\u001b[1;3m Cysticercosis is an infection caused by the larvae of the parasite Taenia solium. It can occur when a person swallows tapeworm eggs, which can happen through contaminated food or water or by putting contaminated fingers in the mouth. It is most commonly found in rural areas of developing countries with poor sanitation, where pigs may have contact with human feces. Diagnosis usually requires MRI or CT brain scans, and treatment may involve medication or surgery. People with cysticercosis should also be tested for intestinal tapeworm infection.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: Some possible treatments for cysticercosis include medication and surgery. It is important to also be tested for intestinal tapeworm infection.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: In order to rule out other conditions similar to botulism, such as Guillain-Barré syndrome, stroke, and myasthenia gravis, what specific diagnostic tests might a physician need to conduct?\u001b[0m\u001b[36;1m\u001b[1;3m A physician may need to conduct a brain scan, spinal fluid examination, nerve conduction test (electromyography, or EMG), and a tensilon test for myasthenia gravis in order to rule out other conditions similar to botulism.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: It is important for a physician to conduct these specific diagnostic tests in order to accurately diagnose and treat botulism.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Baylisascaris infection\u001b[0m\u001b[36;1m\u001b[1;3m Baylisascaris infection is a parasitic infection caused by roundworms found in the intestines of various animals, including raccoons and skunks. It can be prevented by avoiding contact with raccoons and their feces, and cleaning up raccoon latrines promptly. Symptoms in humans can vary depending on the location of the larvae in the body, and diagnosis can be difficult. It is important to seek medical attention if you suspect you have been infected.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: Some specific actions that can be undertaken to prevent the risk of contracting a Baylisascaris infection include avoiding contact with raccoons and their feces, promptly cleaning up raccoon latrines, and seeking medical attention if you suspect you have been infected.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: botulism types\u001b[0m\u001b[36;1m\u001b[1;3m There are five main types of botulism: foodborne, wound, infant, adult intestinal toxemia, and iatrogenic.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: Botulism is not transmitted through lice. It is caused by a toxin produced by the bacteria Clostridium botulinum, which can be found in contaminated food, wounds, and other sources.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Toxoplasmosis\u001b[0m\u001b[36;1m\u001b[1;3m Toxoplasmosis is a disease caused by the Toxoplasma gondii parasite. It is estimated that 22.5% of the population in the United States has been infected with Toxoplasma, and infection rates can reach as high as 40% in some parts of the world. The parasite is typically transmitted through contaminated food, contact with infected animal feces, or congenital transmission from an infected mother to her unborn child. While most healthy individuals do not experience symptoms, pregnant women and those with compromised immune systems are at risk for serious health problems.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: Toxoplasmosis can be transmitted through contaminated food, contact with infected animal feces, or congenital transmission from an infected mother to her unborn child. It is important to take precautions to avoid these sources of transmission, especially for pregnant women and those with compromised immune systems.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? No\n",
            "Final Answer: Toxoplasmosis can be transmitted through contaminated food, contact with infected animal feces, or congenital transmission from an infected mother to her unborn child. It is important to take precautions to avoid these sources of transmission, especially for pregnant women and those with compromised immune systems.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Botulism diagnosis\u001b[0m\u001b[36;1m\u001b[1;3m Special tests such as a brain scan, spinal fluid examination, nerve conduction test, and a tensilon test may be needed to diagnose botulism. Tests for botulinum toxin and bacteria can also be performed at certain laboratories.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: It is important to seek medical attention if you suspect your child may have botulism. A doctor will be able to perform the necessary tests and provide appropriate treatment.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Rabies diagnosis\u001b[0m\u001b[36;1m\u001b[1;3m In animals, rabies is diagnosed using the direct fluorescent antibody (DFA) test, which looks for the presence of rabies virus antigens in brain tissue. In humans, several tests are required.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: Rapid and accurate laboratory diagnosis is essential in order to provide timely and appropriate treatment for rabies, as the disease is almost always fatal once symptoms appear. In humans, diagnosis is typically done through a combination of clinical evaluation, laboratory tests, and history of potential exposure to the virus.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Toxoplasmosis\u001b[0m\u001b[36;1m\u001b[1;3m Toxoplasmosis is a disease caused by the Toxoplasma gondii parasite. It is estimated that 22.5% of the population in the United States has been infected with Toxoplasma, and infection rates can reach as high as 40% in some parts of the world. The parasite is typically transmitted through contaminated food, contact with infected animal feces, or congenital transmission from an infected mother to her unborn child. While most healthy individuals do not experience symptoms, pregnant women and those with compromised immune systems are at risk for serious health problems.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: Toxoplasmosis is a disease caused by a parasite and can be transmitted through contaminated food, contact with infected animal feces, or congenital transmission from an infected mother to her unborn child. It is important to take precautions to avoid these sources of transmission, especially for pregnant women and those with compromised immune systems.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "CorrectnessMetric evaluation:   0%|          | 0/10 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ddafa25f88ca47fb81172779c34ca1aa"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Summary with the results.\n",
        "report.correctness_by_question_type()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "id": "WNLuWCB2YJwO",
        "outputId": "2664b76d-67d7-423c-e980-cb15e9f1fa85"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                     correctness\n",
              "question_type                   \n",
              "complex                      0.0\n",
              "conversational               0.0\n",
              "distracting element          0.5\n",
              "double                       0.0\n",
              "simple                       1.0\n",
              "situational                  0.5"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4c3f5775-b394-4777-9900-91fae600903d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>correctness</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>question_type</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>complex</th>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>conversational</th>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>distracting element</th>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>double</th>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>simple</th>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>situational</th>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4c3f5775-b394-4777-9900-91fae600903d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-4c3f5775-b394-4777-9900-91fae600903d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-4c3f5775-b394-4777-9900-91fae600903d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-f01433ce-f9eb-4b40-b909-26eb05a62d47\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f01433ce-f9eb-4b40-b909-26eb05a62d47')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-f01433ce-f9eb-4b40-b909-26eb05a62d47 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"report\",\n  \"rows\": 6,\n  \"fields\": [\n    {\n      \"column\": \"question_type\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"complex\",\n          \"conversational\",\n          \"situational\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"correctness\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.408248290463863,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.0,\n          0.5,\n          1.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "report.correctness_by_topic()"
      ],
      "metadata": {
        "id": "-ZR-AoQyXBCP",
        "outputId": "e3b4fb9f-33ad-4887-cce3-c7ba880058dd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        }
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                    correctness\n",
              "topic                                          \n",
              "Infectious Diseases and Prevention         0.00\n",
              "Others                                     0.20\n",
              "Parasitic Diseases Treatment               0.75"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fe904226-9a51-4206-ad2a-ed325361f635\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>correctness</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>topic</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Infectious Diseases and Prevention</th>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Others</th>\n",
              "      <td>0.20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Parasitic Diseases Treatment</th>\n",
              "      <td>0.75</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fe904226-9a51-4206-ad2a-ed325361f635')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-fe904226-9a51-4206-ad2a-ed325361f635 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-fe904226-9a51-4206-ad2a-ed325361f635');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-936ee9a6-3dc8-4d45-8984-89523f2634f9\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-936ee9a6-3dc8-4d45-8984-89523f2634f9')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-936ee9a6-3dc8-4d45-8984-89523f2634f9 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"report\",\n  \"rows\": 3,\n  \"fields\": [\n    {\n      \"column\": \"topic\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"Infectious Diseases and Prevention\",\n          \"Others\",\n          \"Parasitic Diseases Treatment\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"correctness\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.3883726732577014,\n        \"min\": 0.0,\n        \"max\": 0.75,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.0,\n          0.2,\n          0.75\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "report.component_scores()"
      ],
      "metadata": {
        "id": "XgHVO7jBYl-3",
        "outputId": "ae885f6d-def6-4042-ffc6-95bc80cd7068",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   score\n",
              "RAG Components          \n",
              "GENERATOR       0.400000\n",
              "RETRIEVER       0.750000\n",
              "REWRITER        0.166667\n",
              "ROUTING         1.000000\n",
              "KNOWLEDGE_BASE  0.250000"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e062923f-94b6-4c50-9663-f5937f1775b2\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>score</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>RAG Components</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>GENERATOR</th>\n",
              "      <td>0.400000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>RETRIEVER</th>\n",
              "      <td>0.750000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>REWRITER</th>\n",
              "      <td>0.166667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ROUTING</th>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>KNOWLEDGE_BASE</th>\n",
              "      <td>0.250000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e062923f-94b6-4c50-9663-f5937f1775b2')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e062923f-94b6-4c50-9663-f5937f1775b2 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e062923f-94b6-4c50-9663-f5937f1775b2');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-1b892c2e-9e29-435a-ae8c-02f79a938811\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1b892c2e-9e29-435a-ae8c-02f79a938811')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-1b892c2e-9e29-435a-ae8c-02f79a938811 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"report\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"RAG Components\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"RETRIEVER\",\n          \"KNOWLEDGE_BASE\",\n          \"REWRITER\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"score\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.35186013635471064,\n        \"min\": 0.16666666666666666,\n        \"max\": 1.0,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.75,\n          0.25,\n          0.16666666666666666\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Obtaining the incorrect answers\n",
        "failures = report.get_failures()[:2]\n",
        "failures"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "EsBU-rL4d1lM",
        "outputId": "b38acc04-4a4b-4a3d-d64c-24569ca63b43"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                               question  \\\n",
              "id                                                                                        \n",
              "83b3c29a-796d-4440-b38b-02a40662f506  In order to rule out other conditions similar ...   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50  What are some specific actions that can be und...   \n",
              "\n",
              "                                                                       reference_answer  \\\n",
              "id                                                                                        \n",
              "83b3c29a-796d-4440-b38b-02a40662f506  Tests for botulism may include a brain scan, s...   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50  Preventative measures include avoiding contact...   \n",
              "\n",
              "                                                                      reference_context  \\\n",
              "id                                                                                        \n",
              "83b3c29a-796d-4440-b38b-02a40662f506  Document 82: Physicians may consider the diagn...   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50  Document 67: Raccoons are the primary, or defi...   \n",
              "\n",
              "                                     conversation_history  \\\n",
              "id                                                          \n",
              "83b3c29a-796d-4440-b38b-02a40662f506                   []   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50                   []   \n",
              "\n",
              "                                                                               metadata  \\\n",
              "id                                                                                        \n",
              "83b3c29a-796d-4440-b38b-02a40662f506  {'question_type': 'complex', 'seed_document_id...   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50  {'question_type': 'complex', 'seed_document_id...   \n",
              "\n",
              "                                                                           agent_answer  \\\n",
              "id                                                                                        \n",
              "83b3c29a-796d-4440-b38b-02a40662f506  It is important for a physician to conduct the...   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50  Some specific actions that can be undertaken t...   \n",
              "\n",
              "                                      correctness  \\\n",
              "id                                                  \n",
              "83b3c29a-796d-4440-b38b-02a40662f506        False   \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50        False   \n",
              "\n",
              "                                                                     correctness_reason  \n",
              "id                                                                                       \n",
              "83b3c29a-796d-4440-b38b-02a40662f506  The agent did not provide the specific diagnos...  \n",
              "65c3ad44-0282-4212-b7a6-1d09becfec50  The agent's answer is partially correct but it...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-54dc2cb6-62f2-442d-a8aa-ddd67f09fc3c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>reference_answer</th>\n",
              "      <th>reference_context</th>\n",
              "      <th>conversation_history</th>\n",
              "      <th>metadata</th>\n",
              "      <th>agent_answer</th>\n",
              "      <th>correctness</th>\n",
              "      <th>correctness_reason</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>83b3c29a-796d-4440-b38b-02a40662f506</th>\n",
              "      <td>In order to rule out other conditions similar ...</td>\n",
              "      <td>Tests for botulism may include a brain scan, s...</td>\n",
              "      <td>Document 82: Physicians may consider the diagn...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'complex', 'seed_document_id...</td>\n",
              "      <td>It is important for a physician to conduct the...</td>\n",
              "      <td>False</td>\n",
              "      <td>The agent did not provide the specific diagnos...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65c3ad44-0282-4212-b7a6-1d09becfec50</th>\n",
              "      <td>What are some specific actions that can be und...</td>\n",
              "      <td>Preventative measures include avoiding contact...</td>\n",
              "      <td>Document 67: Raccoons are the primary, or defi...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'complex', 'seed_document_id...</td>\n",
              "      <td>Some specific actions that can be undertaken t...</td>\n",
              "      <td>False</td>\n",
              "      <td>The agent's answer is partially correct but it...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-54dc2cb6-62f2-442d-a8aa-ddd67f09fc3c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-54dc2cb6-62f2-442d-a8aa-ddd67f09fc3c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-54dc2cb6-62f2-442d-a8aa-ddd67f09fc3c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ef8a1940-c976-4191-be67-ecdb8f75d70c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ef8a1940-c976-4191-be67-ecdb8f75d70c')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ef8a1940-c976-4191-be67-ecdb8f75d70c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_25741bf2-d3e6-41b1-8721-ba0c0f53894d\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('failures')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_25741bf2-d3e6-41b1-8721-ba0c0f53894d button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('failures');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "failures",
              "summary": "{\n  \"name\": \"failures\",\n  \"rows\": 2,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"65c3ad44-0282-4212-b7a6-1d09becfec50\",\n          \"83b3c29a-796d-4440-b38b-02a40662f506\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"question\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"What are some specific actions that can be undertaken to prevent the risk of contracting a Baylisascaris infection, considering the environments where raccoons are most likely to frequent?\",\n          \"In order to rule out other conditions similar to botulism, such as Guillain-Barr\\u00e9 syndrome, stroke, and myasthenia gravis, what specific diagnostic tests might a physician need to conduct?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"reference_answer\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"Preventative measures include avoiding contact with raccoons and their feces, washing hands after working or playing outdoors, not keeping or feeding wild animals as pets, discouraging raccoons from living in and around homes or parks, staying away from areas and materials that might be contaminated by raccoon feces, and cleaning areas near homes that have been identified as raccoon latrines. If cleaning a latrine, wear disposable gloves and a N95-rated respirator, avoid stirring up dust, wear rubber boots or disposable booties, and dispose of feces properly. It's also important to wash hands well after cleaning and to launder clothes thoroughly.\",\n          \"Tests for botulism may include a brain scan, spinal fluid examination, nerve conduction test (electromyography, or EMG), and a tensilon test for myasthenia gravis. Tests for botulinum toxin and for bacteria that cause botulism can also be performed.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"reference_context\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"Document 67: Raccoons are the primary, or definitive, host of Baylisascaris procyonis, a roundworm. Raccoons become infected with Baylisascaris in one of two ways:\\n    \\n     - Young raccoons become infected by eating eggs during foraging, feeding, and grooming.\\n     - Adult raccoons acquire the infection by eating rodents, rabbits, and birds infected with the larvae of Baylisascaris.\\n    \\n    \\nInfected raccoons have been found throughout the United States, mainly in the Midwest, Northeast, Middle Atlantic, and West Coast.\\n    \\nRaccoons are peridomestic animals, which means they live in or around areas where people live. Roundworm eggs are passed in the feces of infected raccoons. Raccoons defecate in communal sites, called latrines. Raccoon latrines are often found at bases of trees, unsealed attics, or on flat surfaces such as logs, tree stumps, rocks, decks, and rooftops. As more raccoons move into populated areas, the number and density of their latrines will increase.\\n    \\nWhile raccoons are the roundworm's primary host, other types of animals can become infected. Birds and small mammals, such as rodents and rabbits, are susceptible to the parasite. Unlike raccoons, these animals sometimes show signs of infection, such as muscle spasms, tremors, and progressive weakness; infection can lead to death. Predator animals, including dogs, may become infected by eating an animal that has been infected with Baylisascaris. In some dogs, Baylisascaris may develop to adult worms and pass eggs in the dogs' feces.\\n    \\nThe worms develop to maturity in the raccoon intestine, where they produce millions of eggs that are passed in the feces. Eggs that are excreted by raccoons are not immediately infectious. These eggs must develop in the environment for 2 to 4 weeks, after which the eggs are able to cause infection. The eggs are resistant to most environmental conditions and with adequate moisture, can survive for years.\\n    \\n     \\n    \\n    \\nHumans become infected by ingesting embryonated (fertile) eggs. Anyone who is exposed to environments where raccoons frequent is potentially at risk. Young children or developmentally disabled persons are at highest risk for infection as they may be more likely to put contaminated fingers, soil, or objects into their mouths.\\n    \\nHunters, trappers, taxidermists, and wildlife handlers may also be at increased risk if they have contact with raccoons or raccoon habitats.\\n    \\nFewer than 25 cases of Baylisascaris disease have been documented in the United States. However, it is possible that some cases are incorrectly diagnosed as other infections or go undiagnosed. Cases that are diagnosed tend to be severe.\\n    \\nCases have been reported in California, Illinois, Louisiana, Massachusetts, Michigan, Minnesota, Missouri, New York, Oregon, and Pennsylvania. As of 2012, there were 16 published human neurological cases in the US; six of the infected persons died.\\n\\nDocument 70: Baylisascaris infection can be prevented by avoiding contact with raccoons and their feces. Washing your hands after working or playing outdoors is good practice for preventing a number of diseases.\\n    \\nDo not keep, feed, or adopt wild animals, including raccoons, as pets. Infection rarely causes symptoms in raccoons, so you cannot tell if a raccoon is infected by observing its behavior. Roundworm eggs passed in the feces of infected raccoons are not visible to the naked eye. Eggs can only be seen using a microscope.\\n    \\n     \\n    \\n    \\nYou may discourage raccoons from living in and around your home or parks by taking these steps:\\n    \\n     - prevent access to food\\n     - keep trash containers tightly closed\\n     - close off access to attics and basements\\n     - keep sandboxes covered when not in use (raccoons may use sandboxes as a latrine)\\n     - remove fish ponds -- they eat the fish and drink the water\\n     - eliminate water sources\\n     - remove bird feeders\\n     - clear brush so raccoons are not likely to make a den on your property\\n    \\n    \\nStay away from areas and materials that might be contaminated by raccoon feces. Raccoons typically defecate at the base of or in raised forks of trees, or on raised horizontal surfaces such as fallen logs, stumps, or large rocks. Raccoon feces also can be found on woodpiles, decks, rooftops, and in attics, garages, and haylofts. Feces usually are dark and tubular, have a pungent odor (usually worse than dog or cat feces), and often contain undigested seeds or other food items.\\n    \\nIf you have found a raccoon latrine near your home, cleaning the area may prevent possible infection. Newly deposited eggs take at least 2-4 weeks to become infective. Prompt removal and destruction of raccoon feces will reduce risk for exposure and possible infection.\\n    \\nMore on: Raccoon Latrine Clean-up [PDF, 111 KB, 1 page]\\n    \\n     \\n    \\n    \\nIf you choose to clean the site yourself, care should be taken to avoid contaminating hands and clothes.\\n    \\n     - Wear disposable gloves to help prevent cross contamination.\\n     - Wear a N95-rated respirator if working in a confined space to prevent accidental ingestion of eggs or other harmful materials.\\n     - Avoid stirring up dust and debris- you can lightly mist the latrine area with a little water from a spray bottle to reduce the amount of dust.\\n     - Wear rubber boots that can be scrubbed or cover your shoes with disposable booties that can be thrown away, so that you do not bring eggs into your household.\\n     - Feces and material contaminated with raccoon feces should be removed and burned, buried, or sent to a landfill.\\n     - Most chemicals do not kill roundworm eggs; however, heat kills the eggs instantly.\\n     - Treat feces-soiled decks, patios, and other surfaces with boiling water or a propane torch (please contact your local fire department for regulations and safety practices).\\n    \\n    \\nTo help further reduce the risk of possible infection, wash your hands well with soap and warm running water. Clean/launder your clothes thoroughly with hot water and detergent.\\n    \\nMore on: Handwashing\\n    \\nIf you are cleaning an indoor raccoon latrine and are not able to use a propane torch, use a damp (but not wet) sponge to wipe the area with hot soapy water. Rinse your sponge frequently. After you are finished, flush dirty water down the toilet. Place the sponge in a plastic bag and put the plastic bag in the garbage.\\n    \\nContact your local animal control office for additional assistance.\\n    Dogs\\n    \\nDogs may be infected with adult B. procyonis roundworms, but may not show symptoms. Have all pets de-wormed under a veterinarian's supervision and take precautions to avoid contact with their feces.\\n    Exotic pets\\n    \\nRaccoons and dogs are not the only hosts of Baylisascaris. B. procyonis infection has also been documented in kinkajous. Other animals such as coatis may be susceptible. When wild animals are kept as pets, there can be a risk of disease transmission to humans.\\n\\nDocument 66: Baylisascaris worms are intestinal parasites found in a wide variety of animals. Different species of Baylisascaris are associated with different animal hosts. For example, Baylisascaris procyonis is found in raccoons and Baylisascaris columnaris is an intestinal parasite found in skunks. Cases of Baylisascaris infection in people are not frequently reported, but can be severe. Baylisascaris procyonis is thought to pose the greatest risk to humans because of the often close association of raccoons to human dwellings.\",\n          \"Document 82: Physicians may consider the diagnosis if the patient's history and physical examination suggest botulism. However, these clues are usually not enough to allow a diagnosis of botulism. Other diseases such as Guillain-Barr\\u00e9 syndrome, stroke, and myasthenia gravis can appear similar to botulism, and special tests may be needed to exclude these other conditions. These tests may include a brain scan, spinal fluid examination, nerve conduction test (electromyography, or EMG), and a tensilon test for myasthenia gravis. Tests for botulinum toxin and for bacteria that cause botulism can be performed at some state health department laboratories and at CDC.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"conversation_history\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"metadata\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"agent_answer\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"Some specific actions that can be undertaken to prevent the risk of contracting a Baylisascaris infection include avoiding contact with raccoons and their feces, promptly cleaning up raccoon latrines, and seeking medical attention if you suspect you have been infected.\",\n          \"It is important for a physician to conduct these specific diagnostic tests in order to accurately diagnose and treat botulism.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"correctness\",\n      \"properties\": {\n        \"dtype\": \"boolean\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          false\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"correctness_reason\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"The agent's answer is partially correct but it misses out on several important preventative measures such as washing hands after working or playing outdoors, not keeping or feeding wild animals as pets, discouraging raccoons from living in and around homes or parks, and the specific precautions to take when cleaning a latrine.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Giskard explains the reasons why it considers the answers to be incorrect.\n",
        "failures['correctness_reason'].iloc[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "id": "VBFOYCMTeOUd",
        "outputId": "20e4f2df-2fbf-485b-b574-4e1a9b8daa08"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"The agent's answer is partially correct but it misses out on several important preventative measures such as washing hands after working or playing outdoors, not keeping or feeding wild animals as pets, discouraging raccoons from living in and around homes or parks, and the specific precautions to take when cleaning a latrine.\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "# Conclusions.\n",
        "The experiment has been a small success. The Vectorial database has been configured and filled with information from the dataset. A LangChain agent has been created, and it has been able to retrieve information from the database only when necessary. Don't forget that our ChatBot has memory.\n",
        "\n",
        "All of this in just a few lines of code!\n"
      ],
      "metadata": {
        "id": "gdyhyd6nJnwN"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ykg5TYA033yR"
      },
      "source": [
        "---"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.9.12"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4b90b500fe274ce3804d69633e3ce045": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f13720f8c0054b37ad63800962c01bd8",
              "IPY_MODEL_a4689ab634be40769ff8c18d3076fbfd",
              "IPY_MODEL_6850ddb41cdb47b3a42117ab825ddf55"
            ],
            "layout": "IPY_MODEL_def97bfc99c843fcaefeec857804e9a7"
          }
        },
        "f13720f8c0054b37ad63800962c01bd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2341ca0e2acc46aab7755fab0442d7ec",
            "placeholder": "​",
            "style": "IPY_MODEL_b6229a3e1b404dc1ae23a13d599f2b1a",
            "value": "README.md: 100%"
          }
        },
        "a4689ab634be40769ff8c18d3076fbfd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce0ebb7981da496982ecb22864c9a8f8",
            "max": 233,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5174e8de97ad4bf8a6634ff749b61dff",
            "value": 233
          }
        },
        "6850ddb41cdb47b3a42117ab825ddf55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_094b862e70a64a8bbe80625e9aacdd77",
            "placeholder": "​",
            "style": "IPY_MODEL_834b7b72458341949259d352a681a2d5",
            "value": " 233/233 [00:00&lt;00:00, 17.1kB/s]"
          }
        },
        "def97bfc99c843fcaefeec857804e9a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2341ca0e2acc46aab7755fab0442d7ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b6229a3e1b404dc1ae23a13d599f2b1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ce0ebb7981da496982ecb22864c9a8f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5174e8de97ad4bf8a6634ff749b61dff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "094b862e70a64a8bbe80625e9aacdd77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "834b7b72458341949259d352a681a2d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c3d158dc007941e0bacf1ada9b4d7959": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_79b2ac0b24bf4dad8babf7b645f8bb84",
              "IPY_MODEL_c933fc46c07440e288dccbd1c349f0a3",
              "IPY_MODEL_1db2aec4c4fe469387537c1ad9eb4137"
            ],
            "layout": "IPY_MODEL_9a7c82cff6374dadb1185d8a5653fe22"
          }
        },
        "79b2ac0b24bf4dad8babf7b645f8bb84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_63fac17cc9f444278a3ed1891fb5367f",
            "placeholder": "​",
            "style": "IPY_MODEL_fc51b487afad4be48fe37f9d5cdf8e09",
            "value": "medDataset_processed.csv: 100%"
          }
        },
        "c933fc46c07440e288dccbd1c349f0a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_44ccb08744c54d0d8b1a369b00110241",
            "max": 22466890,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8aa404c0ddda4b47955ca5b198d63604",
            "value": 22466890
          }
        },
        "1db2aec4c4fe469387537c1ad9eb4137": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a668f16bf584d65a36a1a1f8b32a045",
            "placeholder": "​",
            "style": "IPY_MODEL_81591a2f532547e5ac1db6e900e86c3e",
            "value": " 22.5M/22.5M [00:02&lt;00:00, 8.77MB/s]"
          }
        },
        "9a7c82cff6374dadb1185d8a5653fe22": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "63fac17cc9f444278a3ed1891fb5367f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc51b487afad4be48fe37f9d5cdf8e09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "44ccb08744c54d0d8b1a369b00110241": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8aa404c0ddda4b47955ca5b198d63604": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0a668f16bf584d65a36a1a1f8b32a045": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81591a2f532547e5ac1db6e900e86c3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6455d0b54a6d4c37b8a4c7e35ad170df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a4fb5033245f46d4b9376e38956bd635",
              "IPY_MODEL_4bb13b7ac5c64ae2a12f03e569320542",
              "IPY_MODEL_be209ee684de4813a63e15c5a562c0f1"
            ],
            "layout": "IPY_MODEL_9af4090ffa2f4e178f7abda763e091e0"
          }
        },
        "a4fb5033245f46d4b9376e38956bd635": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ab796022b81b4ee3b4f105f88f722453",
            "placeholder": "​",
            "style": "IPY_MODEL_8051d6bec5864a3f82a3949576d51669",
            "value": "Generating train split: 100%"
          }
        },
        "4bb13b7ac5c64ae2a12f03e569320542": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_16e3ee920a5d4342b93e4e4b8868c700",
            "max": 16407,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_58bc373738074a51bbe73777881d3179",
            "value": 16407
          }
        },
        "be209ee684de4813a63e15c5a562c0f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b13b00d71ff438da830af5d1b9bc7a0",
            "placeholder": "​",
            "style": "IPY_MODEL_4fc9af1de5904111a55f93fe0493d003",
            "value": " 16407/16407 [00:00&lt;00:00, 47610.71 examples/s]"
          }
        },
        "9af4090ffa2f4e178f7abda763e091e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ab796022b81b4ee3b4f105f88f722453": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8051d6bec5864a3f82a3949576d51669": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "16e3ee920a5d4342b93e4e4b8868c700": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58bc373738074a51bbe73777881d3179": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0b13b00d71ff438da830af5d1b9bc7a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4fc9af1de5904111a55f93fe0493d003": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "402bc040234745f8ae9a557728eb8354": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c1409fb9cd114ea1bcc5dd77ddc504e7",
              "IPY_MODEL_1e3b95759e8a46e693599b340a0835f9",
              "IPY_MODEL_4038dab690dc4e17a0123cd8bc07edcd"
            ],
            "layout": "IPY_MODEL_04cdfa01853447449e6e58ae355e0687"
          }
        },
        "c1409fb9cd114ea1bcc5dd77ddc504e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_332819930be54450b2c4c66aa37fb3cd",
            "placeholder": "​",
            "style": "IPY_MODEL_4e03497cf82e44b7bc1135d73e91d4f0",
            "value": "Generating questions: 100%"
          }
        },
        "1e3b95759e8a46e693599b340a0835f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_82b885247c6f4cb2a826b6abc6d5175e",
            "max": 10,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d94c38c92b444e92b6e8a08c4c652e64",
            "value": 10
          }
        },
        "4038dab690dc4e17a0123cd8bc07edcd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ceb438b0126847ff9009604005732812",
            "placeholder": "​",
            "style": "IPY_MODEL_372c3c4bcbf643ce9141a987125d8206",
            "value": " 10/10 [01:23&lt;00:00,  9.44s/it]"
          }
        },
        "04cdfa01853447449e6e58ae355e0687": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "332819930be54450b2c4c66aa37fb3cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e03497cf82e44b7bc1135d73e91d4f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "82b885247c6f4cb2a826b6abc6d5175e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d94c38c92b444e92b6e8a08c4c652e64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ceb438b0126847ff9009604005732812": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "372c3c4bcbf643ce9141a987125d8206": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0f0e0eb678fb4561ae67c8a56d0024f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f94992c7afee4b1591db5164cf107880",
              "IPY_MODEL_9e00a1effe084ab9806fb7aab4775984",
              "IPY_MODEL_cd7396bfa8df48b2864a109ce7c66a2c"
            ],
            "layout": "IPY_MODEL_8f5ef7b50a3c43609c7c1a168e6ac9b6"
          }
        },
        "f94992c7afee4b1591db5164cf107880": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7fce24a65eb74ec8bec35caefedc3107",
            "placeholder": "​",
            "style": "IPY_MODEL_cad8096da6874fe0898f1345c1e7bbf4",
            "value": "Asking questions to the agent: 100%"
          }
        },
        "9e00a1effe084ab9806fb7aab4775984": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3b083846d432457782171a810b460771",
            "max": 10,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ae34ed2aae744624b048ec526413dbdf",
            "value": 10
          }
        },
        "cd7396bfa8df48b2864a109ce7c66a2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6b8a6330b0b74c649176611711b99231",
            "placeholder": "​",
            "style": "IPY_MODEL_f3ad037f417c46b5a3e52e86486c379a",
            "value": " 10/10 [00:35&lt;00:00,  3.66s/it]"
          }
        },
        "8f5ef7b50a3c43609c7c1a168e6ac9b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7fce24a65eb74ec8bec35caefedc3107": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cad8096da6874fe0898f1345c1e7bbf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3b083846d432457782171a810b460771": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ae34ed2aae744624b048ec526413dbdf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6b8a6330b0b74c649176611711b99231": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f3ad037f417c46b5a3e52e86486c379a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ddafa25f88ca47fb81172779c34ca1aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2885e20894324970a12f52d3cd7f15b1",
              "IPY_MODEL_98ffcb3dc9bf4cf6bdb8a00e57f73a55",
              "IPY_MODEL_6f234f8281c4445792dc4ea0ce7a7b9e"
            ],
            "layout": "IPY_MODEL_840a177b18254751b07f564176429e36"
          }
        },
        "2885e20894324970a12f52d3cd7f15b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9806352a846a4fec9c6fecd2aa3fc1ad",
            "placeholder": "​",
            "style": "IPY_MODEL_c024e5f9f06342169b7f7c4a5327d7ff",
            "value": "CorrectnessMetric evaluation: 100%"
          }
        },
        "98ffcb3dc9bf4cf6bdb8a00e57f73a55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a66f8156f0d6406cab082b74b1b74aa2",
            "max": 10,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b023814d0cf04b589926d9f64cd72ad3",
            "value": 10
          }
        },
        "6f234f8281c4445792dc4ea0ce7a7b9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2ec7a5d413864daab9f68dbdd30b04b2",
            "placeholder": "​",
            "style": "IPY_MODEL_33297c2c5e1d4ad0a3fe95d7bff4cb46",
            "value": " 10/10 [00:23&lt;00:00,  2.71s/it]"
          }
        },
        "840a177b18254751b07f564176429e36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9806352a846a4fec9c6fecd2aa3fc1ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c024e5f9f06342169b7f7c4a5327d7ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a66f8156f0d6406cab082b74b1b74aa2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b023814d0cf04b589926d9f64cd72ad3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2ec7a5d413864daab9f68dbdd30b04b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33297c2c5e1d4ad0a3fe95d7bff4cb46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}